Skip to content

[VL] Delta write / Spark 3.2, 3.3 write: Fix null sink memory pool causing crash with remote file systems#11410

Merged
zhztheplayer merged 2 commits intoapache:mainfrom
zhztheplayer:wip-null-sink
Jan 15, 2026
Merged

[VL] Delta write / Spark 3.2, 3.3 write: Fix null sink memory pool causing crash with remote file systems#11410
zhztheplayer merged 2 commits intoapache:mainfrom
zhztheplayer:wip-null-sink

Conversation

@zhztheplayer
Copy link
Member

@zhztheplayer zhztheplayer commented Jan 14, 2026

Fixes the following error:

facebook::velox::filesystems::S3WriteFile::Impl::Impl(std::basic_string_view<char, std::char_traits<char> >, Aws::S3::S3Client*, facebook::velox::memory::MemoryPool*)+0xb6)[0x7f9892b99726]
facebook::velox::filesystems::S3WriteFile::S3WriteFile(std::basic_string_view<char, std::char_traits<char> >, Aws::S3::S3Client*, facebook::velox::memory::MemoryPool*)+0x81)[0x7f9892b973e1]
facebook::velox::filesystems::S3FileSystem::openFileForWrite(std::basic_string_view<char, std::char_traits<char> >, facebook::velox::filesystems::FileOptions const&)+0x54)[0x7f9892b85e64]
facebook::velox::filesystems::s3WriteFileSinkGenerator(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)+0x155)[0x7f9892b83335]
std::_Function_handler<std::unique_ptr<facebook::velox::dwio::common::FileSink, std::default_delete<facebook::velox::dwio::common::FileSink> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&), std::unique_ptr<facebook::velox::dwio::common::FileSink, std::default_delete<facebook::velox::dwio::common::FileSink> > (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)+0x11)[0x7f9891b68111]
facebook::velox::dwio::common::FileSink::create(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)+0x5f)[0x7f9891b668bf]
gluten::VeloxParquetDataSourceS3::initSink(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&)+0x205)[0x7f98916d41a5]
gluten::VeloxParquetDataSource::init(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&)+0x16)[0x7f9891774266]
Java_org_apache_gluten_datasource_VeloxDataSourceJniWrapper_init+0x370)[0x7f98916f5d70]

This fix is by @FelixYBW.

…using crash

Fixes the following error:

```
facebook::velox::filesystems::S3WriteFile::Impl::Impl(std::basic_string_view<char, std::char_traits<char> >, Aws::S3::S3Client*, facebook::velox::memory::MemoryPool*)+0xb6)[0x7f9892b99726]
facebook::velox::filesystems::S3WriteFile::S3WriteFile(std::basic_string_view<char, std::char_traits<char> >, Aws::S3::S3Client*, facebook::velox::memory::MemoryPool*)+0x81)[0x7f9892b973e1]
facebook::velox::filesystems::S3FileSystem::openFileForWrite(std::basic_string_view<char, std::char_traits<char> >, facebook::velox::filesystems::FileOptions const&)+0x54)[0x7f9892b85e64]
facebook::velox::filesystems::s3WriteFileSinkGenerator(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)+0x155)[0x7f9892b83335]
std::_Function_handler<std::unique_ptr<facebook::velox::dwio::common::FileSink, std::default_delete<facebook::velox::dwio::common::FileSink> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&), std::unique_ptr<facebook::velox::dwio::common::FileSink, std::default_delete<facebook::velox::dwio::common::FileSink> > (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)+0x11)[0x7f9891b68111]
facebook::velox::dwio::common::FileSink::create(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, facebook::velox::dwio::common::FileSink::Options const&)+0x5f)[0x7f9891b668bf]
gluten::VeloxParquetDataSourceS3::initSink(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&)+0x205)[0x7f98916d41a5]
gluten::VeloxParquetDataSource::init(std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&)+0x16)[0x7f9891774266]
Java_org_apache_gluten_datasource_VeloxDataSourceJniWrapper_init+0x370)[0x7f98916f5d70]
```
@github-actions github-actions bot added the VELOX label Jan 14, 2026
@zhztheplayer zhztheplayer marked this pull request as draft January 14, 2026 11:05
@zhztheplayer zhztheplayer marked this pull request as ready for review January 14, 2026 12:57
@zhztheplayer zhztheplayer merged commit c571c20 into apache:main Jan 15, 2026
64 checks passed
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants